package com.my_test.webmagic.webmagic_modules;

import com.alibaba.fastjson.JSONObject;
import com.my_test.webmagic.module.entity.*;
import com.my_test.webmagic.module.service.*;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.annotation.Resource;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * @author zhangyalin@act-telecom.com
 * @date 2019-05-23 10:13
 **/
@Component
public class WebMagicProcessor implements PageProcessor {

    @Resource
    private TiebaTypeService tieziTypeService;
    @Resource
    private TieziService tieziService;
    @Resource
    private UserService userService;
    @Resource
    private FloorService floorService;
    @Resource
    private FloorLoService floorLoService;
    @Resource
    private TiebaService tiebaService;
    @Resource
    private TransportClient transportClient;


    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8");
    //匹配类型
    public final static String typeRegex = "http://tieba.baidu.com/f/index/forumclass";
    //匹配贴吧
    public final static String tiebaRegex = "(http[s]?://tieba.baidu.com/f\\?[\\S]*kw=[\\S]+)";
    //匹配大类型 http://tieba.baidu.com/f/index/forumpark?pcn=娱乐明星&pci=0&ct=1&rn=20&pn=1
    public final static String type1Regex = "(http[s]?://tieba.baidu.com/f/index/forumpark\\?pcn=[\\S]+)";
    //匹配小类型 http://tieba.baidu.com/f/index/forumpark?cn=%E6%B8%AF%E5%8F%B0%E4%B8%9C%E5%8D%97%E4%BA%9A%E6%98%8E%E6%98%9F&ci=0&pcn=%E5%A8%B1%E4%B9%90%E6%98%8E%E6%98%9F&pci=0&ct=1
    public final static String type2Regex = "(http[s]?://tieba.baidu.com/f/index/forumpark\\?cn=[\\S]+)";
    //匹配帖子 https://tieba.baidu.com/p/613zi9422240
    public final static String tieziRegex = "(http[s]?://tieba.baidu.com/p/[\\S]+)";

    @Override
    public void process(Page page) {

        String url = page.getUrl().get();
        System.out.println(url);
        List<String> all = page.getHtml().links().all();
        List<String> regexAll = new ArrayList<>();
        if (url.matches(typeRegex)) {
            //查找帖吧 一级 二级分类
            List<String> oneLevel = new ArrayList<>();
            List<String> twoLevel = new ArrayList<>();
            for (String s : all) {
                if (s.matches(type1Regex)) {
                    oneLevel.add(s);
                } else if (s.matches(type2Regex)) {
                    twoLevel.add(s);
                }
            }
            page.addTargetRequests(twoLevel);
            //拼装入库
            List<TiebaType> typeList = new ArrayList<>();
            TiebaType type = new TiebaType();
            String one;
            String two;
            try {
                for (String s : oneLevel) {
                    s = URLDecoder.decode(s, "utf8");
                    one = s.substring(s.indexOf("pcn=") + 4, s.indexOf("&pci="));
                    type = new TiebaType();
                    type.setType(one);
                    typeList.add(type);
                }
                for (String s : twoLevel) {
                    s = URLDecoder.decode(s, "utf8");
                    one = s.substring(s.indexOf("pcn=") + 4, s.indexOf("&pci="));
                    two = s.substring(s.indexOf("cn=") + 3, s.indexOf("&ci="));
                    type = new TiebaType();
                    type.setType(two);
                    type.setpType(one);
                    typeList.add(type);
                }
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            page.putField("entityList", typeList);
//            tieziTypeService.saveBatch(typeList);

        } else if (url.matches(tiebaRegex)) {
            //获取贴吧唯一标示

            try {
                url = URLDecoder.decode(url, "utf8");
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            String mark = url.contains("&") ? url.substring(url.indexOf("kw=") + 3, url.indexOf("&")) : url.substring(url.indexOf("kw=") + 3);
           /* if (url.contains("pn=0") || !url.contains("pn=")) {
                //获取贴吧类型
                Document document = page.getHtml().getDocument();
                Elements elementsByClass = document.getElementsByClass("forum_dir_info bottom_list clearfix");
                String typeAttrUrl = elementsByClass.get(0).getElementsByAttributeValue("rel", "noreferrer").attr("href");
                String type = typeAttrUrl.substring(typeAttrUrl.indexOf("sd="));

            }*/

            Elements tieziElements = page.getHtml().getDocument().getElementsByClass(" j_thread_list clearfix");
            if (tieziElements != null) {
                List<Tiezi> tieziList = new ArrayList<>();
                Tiezi tiezi;
                for (Element tieziElement : tieziElements) {
                    String tieziJsonString = tieziElement.attr("data-field");
                    //反序列化为帖子对象，入库
                    JSONObject jsonObject = JSONObject.parseObject(tieziJsonString);
                    tiezi = new Tiezi();
                    tiezi.setTiebaMark(mark);
                    tiezi.setTitle(tieziElement.getElementsByClass("threadlist_title").get(0).getElementsByTag("a").attr("title"));
                    tiezi.setTieziId((long) jsonObject.get("id"));
                    tiezi.setAuthorName((String) jsonObject.get("author_name"));
                    tiezi.setAuthorNickname((String) jsonObject.get("author_nickname"));
                    tiezi.setAuthorPortrait((String) jsonObject.get("author_portrait"));
                    if (jsonObject.get("first_post_id") != null) {
                        tiezi.setFirstPostId((long) jsonObject.get("first_post_id"));
                    }
                    if (jsonObject.get("reply_num") != null) {
                        tiezi.setReplyNum((Integer) jsonObject.get("reply_num"));
                    }
                    if (jsonObject.get("is_bakan") != null) {
                        tiezi.setBakan((boolean) jsonObject.get("is_bakan"));
                    }
                    tiezi.setVid((String) jsonObject.get("vid"));
                    if (jsonObject.get("is_good") != null) {
                        tiezi.setGood((boolean) jsonObject.get("is_good"));
                    }
                    if (jsonObject.get("is_top") != null) {
                        tiezi.setTop((boolean) jsonObject.get("is_top"));
                    }
                    if (jsonObject.get("is_protal") != null) {
                        tiezi.setProtal((boolean) jsonObject.get("is_protal"));
                    }
                    if (jsonObject.get("is_membertop") != null) {
                        tiezi.setMembertop((boolean) jsonObject.get("is_membertop"));
                    }
                    if (jsonObject.get("is_multi_forum") != null) {
                        tiezi.setMultiForum((boolean) jsonObject.get("is_multi_forum"));
                    }

                    tieziList.add(tiezi);
                    //获取帖子链接 ，取tieziJsonString中id属性 /p/

                }
                if (tieziList.size() != 0) {
                   /* tieziService.saveBatch(tieziList);
                    Map<String, Object> o;
                    for (TieziEntity tieziEntity : tieziList) {
                        o = (Map<String, Object>) JSONObject.toJSON(tieziEntity);
                        transportClient.prepareIndex("tiezi", mark, tieziEntity.getTieziId() + "").setSource(o).get();
                    }*/
                }
                page.putField("entityList", tieziList);

            }

            Element pp = page.getHtml().getDocument().getElementById("frs_list_pager");
            if (pp != null) {
                Elements nextPage = pp.getElementsByClass("next");
                if (nextPage != null && nextPage.size() != 0) {//表示还有下一页
                    String href = nextPage.get(0).attr("href");
                    if (href.startsWith("//")) {
                        href = "http:" + href;
                    }
                    page.addTargetRequest(href);
                }
            }

            //当前页所有的帖子
            regexAll = all.stream().filter(s -> s.matches(tieziRegex)).collect(Collectors.toList());
            page.addTargetRequests(regexAll);
            //帖子关联贴吧唯一标示mark， 存入数据库，帖子唯一标示：regexAll 中url /p/后面的值

        } else if (url.matches(tieziRegex)) {
            //查找得是帖子中的楼层数据
            Elements floorElements = page.getHtml().getDocument().getElementsByClass("l_post_bright");
            List<Floor> floorList = new ArrayList<>();
            Floor floor;
            List<User> userList = new ArrayList<>();
            User user;
            for (Element floorElement : floorElements) {
                //楼层数据
                String floorJsonString = floorElement.attr("data-field");
                JSONObject jsonObject = JSONObject.parseObject(floorJsonString);
                if (jsonObject != null) {
                    floor = new Floor();
                    JSONObject authorObject = (JSONObject) jsonObject.get("author");
                    JSONObject contentObject = (JSONObject) jsonObject.get("content");

                    Object userId = authorObject.get("user_id");
                    floor.setAuthorUserId(userId instanceof Long ? (long) userId : userId instanceof Integer ? (long) (int) userId : null);
                    floor.setAuthorUserName((String) authorObject.get("user_name"));
                    floor.setAuthorProps(JSONObject.toJSONString(authorObject.get("props")));
                    floor.setAuthorPortrait((String) authorObject.get("portrait"));
                    floor.setAuthorUserNickname((String) authorObject.get("user_nickname"));
                    user = new User();
                    user.setUserId(floor.getAuthorUserId());
                    user.setUserName(floor.getAuthorUserName());
                    user.setProps(floor.getAuthorProps());
                    user.setPortrait(floor.getAuthorPortrait());
                    user.setUserNickname(floor.getAuthorUserNickname());
                    userList.add(user);

                    if (contentObject.get("post_id") != null) {
                        floor.setPostId((long) contentObject.get("post_id"));
                    }
                    if (contentObject.get("is_anonym") != null) {
                        floor.setAnonym((boolean) contentObject.get("is_anonym"));
                    }
                    if (contentObject.get("forum_id") != null) {
                        Object forumId = contentObject.get("forum_id");
                        floor.setForumId(forumId instanceof Long ? (long) forumId : forumId instanceof Integer ? (long) (int) forumId : null);
                    }
                    if (contentObject.get("thread_id") != null) {
                        floor.setThreadId((long) contentObject.get("thread_id"));
                    }

                    floor.setContent((String) contentObject.get("content"));

                    if (contentObject.get("post_no") != null) {
                        Object postNo = contentObject.get("post_no");
                        floor.setPostNo(postNo instanceof Long ? (long) postNo : postNo instanceof Integer ? (long) (int) postNo : null);
                    }
                    floor.setType((String) contentObject.get("type"));
                    if (contentObject.get("comment_num") != null) {
                        floor.setCommentNum((int) contentObject.get("comment_num"));
                    }
                    if (contentObject.get("is_fold") != null) {
                        floor.setIsFold((int) contentObject.get("is_fold"));

                    }
                    floor.setProps((String) contentObject.get("props"));

                    if (contentObject.get("post_index") != null) {
                        floor.setPostIndex((int) contentObject.get("post_index"));
                    }
                    if (contentObject.get("pb_tpoint") != null) {
                        floor.setPbTpoint((int) contentObject.get("pb_tpoint"));
                    }
                    floorList.add(floor);
                }

                //楼中楼数据
            }
            if (floorList.size() != 0) {
               /* floorService.saveBatch(floorList);
                Map<String, Object> o;
                for (FloorEntity floorEntity : floorList) {
                    o = (Map<String, Object>) JSONObject.toJSON(floorEntity);
                    transportClient.prepareIndex("floor", floorEntity.getThreadId() + "", floorEntity.getPostId() + "").setSource(o).get();
                }*/
            }
            page.putField("entityList", floorList);
            page.putField("userList", userList);

        } else if (url.matches(type2Regex)) {
            //http://tieba.baidu.com/f/index/forumpark?cn=%E4%B8%A5%E8%82%83%E5%B0%8F%E8%AF%B4&ci=0&pcn=%E6%96%87%E5%AD%A6&pci=0&ct=1&rn=20&pn=1
            //查找得是当前类型下所有的贴吧 首页数据,从url中解析出贴吧类型
            List<Tieba> tiebaList = new ArrayList<>();
            try {
                url = URLDecoder.decode(url, "utf8");

                Tieba tieba;
                String sn = url.substring(url.indexOf("cn=") + 3, url.indexOf("&ci="));
                String psn = url.substring(url.indexOf("&pcn=") + 5, url.indexOf("&pci="));

                //找出所有的贴吧

                for (String s : all) {
                    if (s.matches(tiebaRegex)) {
                        regexAll.add(s);
                        tieba = new Tieba();
                        tieba.setType(sn);
                        tieba.setpType(psn);
                        s = URLDecoder.decode(s, "utf8");
                        tieba.setTitle(s.substring(s.indexOf("?kw=") + 4));
                        tieba.setUrl(s);
                        tiebaList.add(tieba);
                    }
                }
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            if (tiebaList.size() != 0) {
               /* tiebaService.saveBatch(tiebaList);
                Map<String, Object> o;
                for (TiebaEntity tiebaEntity : tiebaList) {
                    o = (Map<String, Object>) JSONObject.toJSON(tiebaEntity);
                    transportClient.prepareIndex("tieba", tiebaEntity.getpType(), tiebaEntity.getTitle()).setSource(o).get();
                }*/
            }
            page.putField("entityList", tiebaList);
            page.addTargetRequests(regexAll);
        }

    }


    @Override
    public Site getSite() {
        return site;
    }


    public boolean isIndexExist(String index) {
        return transportClient.admin().indices().prepareExists(index).execute().actionGet().isExists();
    }

    public boolean isTypeExist(String index, String type) {
        return isIndexExist(index) && transportClient.admin().indices().prepareTypesExists(index).setTypes(type).execute().actionGet().isExists();
    }

    public boolean addIndexAndType() throws IOException {
        String index = "ahut";
        String type = "goods";
        // 创建索引映射,相当于创建数据库中的表操作
        CreateIndexRequestBuilder cib = transportClient.admin().indices().prepareCreate(index);
        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties") // 设置自定义字段
                .startObject("goodsName").field("type", "string").endObject() // 商品名称
                .startObject("goodsPrice").field("type", "double").endObject()// 商品价格
                .startObject("goodsUser").field("type", "string").endObject()// 商品主人
                .startObject("goodsTime").field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss").endObject() // 商品上架时间
                .endObject().endObject();
        cib.addMapping(type, mapping);
        return cib.execute().actionGet().isAcknowledged();
    }


    public static void main(String[] args) {

        Map<String, Object> map = new HashMap<>();
        Integer s = new Integer(1231241241);
        map.put("s", s);
        Long a = (long) (int) map.get("s");
        System.out.println(a);
    }


}
